package com.coderfans.csdn;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * Created by ${fanchunshuai} on 2016-7-28.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2016 58到家-版权所有
 */
public class CsdnPageProcessor implements PageProcessor {
    int i =0;
    private Site site = Site.me()
            .setRetryTimes(4)
            .setSleepTime(2000)
            .setCharset("UTF-8");
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(http://blog.csdn\\.net/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(http://blog.csdn\\.net/\\w+)").all());
        List<String> hrefOfA = page.getHtml().xpath("//a/@href").all();
        for(String sq: hrefOfA){
            System.out.println("href:"+(i++)+" | "+sq);
        }

    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new CsdnPageProcessor()).addUrl("http://blog.csdn.net").thread(5).run();

    }
}
